#coding:utf-8

import re
import requests
from bs4 import BeautifulSoup
import json

user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202.94 Chrome/62.0.3202.94 Safari/537.36'
headers={'User-Agent':user_agent}
r = requests.get('http://www.17k.com/list/2712385.html', headers=headers)

# print r.text

soap = BeautifulSoup(r.text, 'html.parser', from_encoding='utf-8')

list = []
for items in soap.find_all(class_="ellipsis"):
    sub_title =  items.string
    sub_href = 'http://www.17k.com' + items.parent.get('href')
    list.append({'href':sub_href,'box_title':sub_title})
    # print sub_href

with open('xiaoshuo.txt', 'wb') as fp:
    json.dump(list,fp=fp, indent=4)